"""
Case Type   : 极致RTO-数据库系统类
Case Name   : 开启极致RTO，业务压力后进行数据导入导出过程中，主机异常挂住后恢复；导出完成后，关闭极致RTO，主备数据一致
Create At   : 2020/10/12
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、进行gs_dump
    3、进行gs_restore
    4、主机数据库进程挂住kill - 19；查询主机数据数据进程状态
    5、查询主备状态
    6、restore进程是否异常
    7、等待10S，恢复主机数据库进程kill - 18；查询主机数据数据进程状态
    8、查询主备状态
    9、restore进程是否异常
    10、restore执行完成后，关闭极致RTO
    11、查询主备数据是否一致
Expect      :
    1、开启极致RTO成功
    2、进行gs_dump功能正常
    3、进行gs_restore运行开始
    4、主机数据库进程挂住kill - 19；查询主机数据数据进程状态
    5、查询主备状态正常
    6、restore进程未异常
    7、等待10S，恢复主机数据库进程kill - 18；查询主机数据数据进程状态正常
    8、查询主备状态正常
    9、restore进程未异常
    10、restore执行完成后，关闭极致RTO成功
    11、查询主备数据一致
History     :
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-3-9 by 5328113;修改dump文件路径；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified:2020-11-18 by 5328113;修改kill-18后的断言；可能存在Rl状态；
    modified:2020-12-14 by 5328113;删除kill-19、kill -18后的断言；变化的状态可能很多状态变化时间也不一定，只需保证后续恢复后功能正常即可；
    modified:2020-12-14 by 5328113;修改restore指定新数据库，以防用例失败影响tpcc库及tpcc运行；teardown增加异常中断kill -18恢复操作
    modified:2020-12-24 by 5328113;修改kill-18 恢复故障后，restore操作未完成，此时备机状态可能会非normal，去掉相关断言
    modified:2020-12-30 by 5328113;修改kill-19后的状态断言
    modified:2021-1-28 by 5328113;tpcc执行完成增加主备同步，删除start成功断言，删除不必要tpcc
    modified:2024-7-10 by peilinqian;查询状态增加断言
"""

import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import *
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_RTO_Dbsys_Case012:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        dump_file_name = 'test_pid.tar'
        self.parent_path = os.path.dirname(macro.DB_INSTANCE_PATH)
        self.file_name = os.path.join(self.parent_path, dump_file_name)

        self.log.info("-----创建restore数据库----")
        self.new_db_name = 'restore_db'
        self.create_db_cmd = f'drop database if exists {self.new_db_name};' \
            f'create database {self.new_db_name};'
        self.drop_db_cmd = f'drop database {self.new_db_name};'
        create_db_result = self.pri_sh.executDbSql(self.create_db_cmd)
        self.log.info(create_db_result)
        self.assertIn(self.constant.CREATE_DATABASE_SUCCESS_MSG,
                      create_db_result)
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、进行gs_dump----')
            dump_result = self.pri_sh.dumpFile(self.file_name,
                                               get_detail=False)
            self.log.info(dump_result)
            self.assertTrue(dump_result)

            self.log.info('----3、进行gs_restore----')
            restore_thread = ComThread(self.pri_sh.restoreFile,
                                       args=(self.file_name,
                                             '',
                                             True,
                                             self.new_db_name,))
            restore_thread.setDaemon(True)
            restore_thread.start()
            time.sleep(3)

            self.log.info('----4、主机数据库进程挂住kill - 19；----')
            self.com.kill_pid_keyword('M primary', 19, self.pri_dbuser)
            time.sleep(5)
            self.log.info('----查询主机数据库进程状态----')
            db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
            self.log.info(db_process_cmd)
            db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
            self.log.info(db_process_result)

            self.log.info('----5、查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('detail')
            self.assertIn(self.constant.standby_need_repair, status)

            self.log.info('----6、restore进程是否异常----')
            restore_ps_cmd = 'ps ux | grep gs_restore | grep -v grep'
            self.log.info(restore_ps_cmd)
            restore_ps_result = self.pri_dbuser.sh(restore_ps_cmd).result()
            self.log.info(restore_ps_result)

            self.log.info('----7、等待10S，恢复主机数据库进程kill - 18；----')
            time.sleep(10)
            self.com.kill_pid_keyword('M primary', 18, self.pri_dbuser)
            time.sleep(1)
            self.log.info('----查询主机数据库进程状态----')
            db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
            self.log.info(db_process_cmd)
            db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
            self.log.info(db_process_result)

            self.log.info('----8、查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

            self.log.info('----9、restore进程是否异常----')
            restore_ps_cmd = 'ps ux | grep gs_restore | grep -v grep'
            self.log.info(restore_ps_cmd)
            restore_ps_result = self.pri_dbuser.sh(restore_ps_cmd).result()
            self.log.info(restore_ps_result)

            # 等待restore执行完成
            restore_thread.join()
            restore_result = restore_thread.get_result()
            self.assertNotIn(self.constant.RESTORE_FAIL_MSG, restore_result)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.max_time)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')

        finally:
            self.log.info('----10、restore执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----11、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

    def tearDown(self):
        self.log.info('----异常中断时，恢复数据库进程kill - 18----')
        self.log.info('----查询主机数据库进程状态----')
        db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
        self.log.info(db_process_cmd)
        db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
        self.log.info(db_process_result)

        self.com.kill_pid_keyword('M primary', 18, self.pri_dbuser)
        time.sleep(1)

        self.log.info('----查询主机数据库进程状态----')
        db_process_cmd = 'ps ux | grep \'M primary\' | grep -v grep'
        self.log.info(db_process_cmd)
        db_process_result = self.pri_dbuser.sh(db_process_cmd).result()
        self.log.info(db_process_result)

        self.log.info('----删除tar文件----')
        self.pri_dbuser.sh(f'rm -rf {self.file_name}').result()
        self.log.info(self.pri_dbuser.sh(f'ls -l {self.file_name}').result())

        self.log.info("----删除restore库----")
        drop_db_result = self.pri_sh.executDbSql(self.drop_db_cmd)
        self.log.info(drop_db_result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info('----Opengauss_Reliability_RTO_Dbsys_Case012:执行完毕----')
